package net.lainiao.reptile.crawlers;

import cn.wanghaomiao.seimi.annotation.Crawler;
import cn.wanghaomiao.seimi.def.BaseSeimiCrawler;
import cn.wanghaomiao.seimi.http.HttpMethod;
import cn.wanghaomiao.seimi.struct.Request;
import cn.wanghaomiao.seimi.struct.Response;
import cn.wanghaomiao.xpath.model.JXDocument;
import net.lainiao.base.model.ReptileTopic;
import net.lainiao.base.model.ReptilemodelinfosWithBLOBs;
import net.lainiao.base.model.Reptilemodels;
import net.lainiao.reptile.*;
import net.lainiao.service.service.ReptileTopicService;
import net.lainiao.service.service.ReptilemodeService;
import net.lainiao.service.service.ReptilemodeinfoService;
import org.apache.commons.lang3.StringUtils;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

/**
 * Created by Administrator on 2017/1/10.
 */
@Crawler(delay = 5, name = "reptileCrawlers", useCookie = true, useUnrepeated = false)
public class ReptileCrawlers extends BaseSeimiCrawler {
    private ReptileTopicService reptileTopicService;
    private ReptilemodeService reptilemodeService;
    private ReptilemodeinfoService reptilemodeinfoService;
    private RedisHelper redisHelper;
    @Override
    public String[] startUrls() {
        return null;
    }

    @Override
    public List<Request> startRequests() {
        reptilemodeService = ApplicationCommon.applicationContext.getBean(ReptilemodeService.class);
        reptileTopicService = ApplicationCommon.applicationContext.getBean(ReptileTopicService.class);
        reptilemodeinfoService = ApplicationCommon.applicationContext.getBean(ReptilemodeinfoService.class);
        redisHelper = ApplicationCommon.applicationContext.getBean(RedisHelper.class);
        List<Request> requests = getRequests();
        List<ReptileTopic> reptileTopics = reptileTopicService.getALL();
        for (ReptileTopic reptileTopic : reptileTopics) {
            HashMap<String, String> hashMap = new HashMap<>();
            hashMap.put("1", reptileTopic.getId().toString());
            hashMap.put("url", reptileTopic.getUrl());
            Request request = new Request(reptileTopic.getUrl(), "analyRes", HttpMethod.GET, null, hashMap);
            requests.add(request);
        }

        for(Request request:requests){
            redisHelper.addProcessed(request.getUrl());
        }

        return requests;
    }

    private List<Request> getRequests() {
        List<Request> requests = new ArrayList<>();
        repeatGetList(requests);
        while (requests.size()<3&&reptilemodeService.geCount()>3){
            repeatGetList(requests);
        }
        return requests;
    }

    private void repeatGetList(List<Request> requests){
        List<Reptilemodels> reptilemodelses = reptilemodeService.getNoAccess();
        for (Reptilemodels reptilemodels : reptilemodelses) {
            if(FilterHelper.isAllow(reptilemodels.getLink(),reptilemodels.getTid())){
                HashMap<String, String> hashMap = new HashMap<>();
                String value = reptilemodels.getId() + "_" + reptilemodels.getTid();
                hashMap.put("2", value);
                hashMap.put("url", reptilemodels.getLink());
                Request request = new Request(reptilemodels.getLink(), "analyRes", HttpMethod.GET, null, hashMap);
                requests.add(request);
            }
            else{
                reptilemodeService.setAccess(reptilemodels.getId());
            }
        }
    }

    @Override
    public void start(Response response) {
        analyRes(response);
    }

    public void analyRes(Response response) {
        try {
            Map<String, String> hashMap = response.getMeta();
            JXDocument doc = response.document();
            int tid = 0;
            if (hashMap.containsKey("1")) {
                tid = Integer.parseInt(hashMap.get("1"));
            } else {
                String resUrl = response.getUrl();
                String value = hashMap.get("2");
                String[] values = value.split("_");
                int id = Integer.parseInt(values[0]);
                tid = Integer.parseInt(values[1]);
                reptilemodeService.setAccess(id);
                if (doc != null) {
                    List<Object> objs = doc.sel("//allText()");
                    List<Object> tils=doc.sel("//title/text()");
                    String title="";
                    if(tils!=null&&tils.size()>0){
                        title=tils.get(0).toString();
                    }
                    if (objs != null && objs.size() > 0) {
                        ReptilemodelinfosWithBLOBs reptilemodelinfos = new ReptilemodelinfosWithBLOBs();
                        if(!redisHelper.isProcessed2(resUrl)){
                            reptilemodelinfos.setLink(resUrl);
                            reptilemodelinfos.setTid(tid);
                            reptilemodelinfos.setHttpcode("200");
                            if(objs.get(0)!=null){
                                String content = objs.get(0).toString();
                                reptilemodelinfos.setContent(content.getBytes());
                                reptilemodelinfos.setTitle(title.getBytes());
                                reptilemodeinfoService.save(reptilemodelinfos);
                            }
                            redisHelper.addProcessed2(resUrl);
                        }
                    }
                }
            }
            if (doc != null) {
                List<Object> urls = doc.sel("//a/@href");
                if (urls != null && urls.size() > 0) {
                    List<Reptilemodels> reptilemodelses = new ArrayList<>();
                    for (Object urlObj : urls) {
                        String url = urlObj.toString();
                        if (!StringUtils.isBlank(url)) {
                            String comUrl = UrlHelper.getUrl(url, response.getUrl());
                            if (FilterHelper.isAllow(comUrl, tid)) {
                                if(!redisHelper.isProcessed1(comUrl)){
                                    Reptilemodels reptileModels = new Reptilemodels();
                                    reptileModels.setTid(tid);
                                    reptileModels.setLink(comUrl);
                                    reptilemodelses.add(reptileModels);
                                    redisHelper.addProcessed1(comUrl);
                                }
                            }
                        }
                    }
                    reptilemodeService.adds(reptilemodelses);
                }
            }
            addNewRequest();
        } catch (Exception e) {
            e.printStackTrace();
            logger.error("{}",e);
        }
    }

    public void addNewRequest() {
        List<Request> requests = getRequests();
        if(ReptileService.status!=0){
            for (Request req : requests) {
                push(req);
                redisHelper.addProcessed(req.getUrl());
            }
        }
    }
}
